# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
import json
import pymongo
import redis
from itemadapter import ItemAdapter

class FbPipeline:
    def process_item(self, item, spider):
        r_con = redis.StrictRedis(db = 2)
        m_con = pymongo.MongoClient()
        m_db = m_con['ff']
        m_tb = m_con['ddw']

        while True:
            try:
                s,data = r_con.blpop('dd:items')
                datas = json.loads(data.decode('utf-8'))
                print(datas)
                if isinstance(datas,dict):
                    m_tb.inser(datas)
                else:
                    print('数据格式有问题')
                    break
            except Exception as e:
                print('数据同步失败%s'%e)
                break